#!/bin/bash

hadoop fs -mkdir /user/jaubertb/twitter

# Download EDF
mkdir -p ./edf; rm -rf ./edf/*;
wget --recursive --mirror -np -A.html -i twitter.txt -Q1000m -P ./edf
hadoop fs -rm -r /user/jaubertb/twitter/edf
hadoop fs -put ./edf /user/jaubertb/twitter
rm -rf edf
# Download football
mkdir -p ./football; rm -rf ./football/*;
wget --recursive --mirror -np -A.html --level=2 -i twitter.txt -Q1000m -P ./football
hadoop fs -rm -r /user/jaubertb/twitter/football
hadoop fs -put ./football /user/jaubertb/twitter
rm -rf football